import pandas as pd
import os
import re

file_list = []


def find_csv(path, filelist):
    dirs = os.listdir(path)
    for filename in dirs:
        if re.match(r"sjz_\w+.csv", filename):
            filelist.append(filename)


def data_cleaning(path, filelist):
    sum_df = pd.DataFrame()
    for filename in filelist:
        df = pd.read_csv(path + filename)
        data = df.drop(
            labels=["Unnamed: 0", "href", "title", "property_name", "property_devname", "property_average_price"],
            axis=1)
        data = data.replace('\t', '', regex=True).replace('\n', '', regex=True).replace(" ", '', regex=True)
        data = data.dropna(axis=0, how='any')
        data['property_year'] = data['property_year'].apply(lambda x: str(x).replace('年建造', ''))
        data['property_area'] = data['property_area'].apply(lambda x: str(x).replace('㎡', ''))
        data['property_price'] = data['property_price'].apply(lambda x: str(x).replace('万', ''))
        # data['property_average_price'] = data['property_average_price'].apply(lambda x: str(x).replace('元/㎡', ''))
        data['property_bedroom'] = data['property_info'].apply(lambda x: x[0])
        data['property_livingroom'] = data['property_info'].apply(lambda x: x[2])
        data['property_lavatory'] = data['property_info'].apply(lambda x: x[4])
        data['property_height'] = data['property_floor'].apply(lambda x: re.search(r'\d+', x).group())
        data['property_floor'] = data['property_floor'].apply(lambda x: x[:x.find('(')] if x.find('(') != -1 else '低层')
        data['property_loc'] = data['property_loc'].apply(lambda x: str(x).split('-', 1)[0])
        data['property_price'] = data['property_price'].apply(lambda x: float(x) * 10000)
        data = data.drop(labels=["property_info"], axis=1)
        # 离散化变量进行独热编码
        # data =pd.get_dummies(data, columns=['property_orientation', 'property_floor', 'property_loc'],prefix_sep='_',dummy_na=False,drop_first=False)
        data.to_csv(f'./data/modify_{filename}')
        print(data.info())
        sum_df = sum_df.append(data, ignore_index=True)

    sum_df = pd.get_dummies(sum_df, columns=['property_orientation', 'property_floor', 'property_loc'], prefix_sep='_',
                            dummy_na=False, drop_first=False)
    sum_df.to_csv(f'./data/SJZ_Property.csv')


def data_statistic(path, filelist):
    sum_df = pd.DataFrame()
    for filename in filelist:
        df = pd.read_csv(path + filename)
        data = df.drop(labels=["Unnamed: 0"], axis=1)
        data = data.replace('\t', '', regex=True).replace('\n', '', regex=True).replace(" ", '', regex=True)
        data = data.dropna(axis=0, how='any')
        data['property_year'] = data['property_year'].apply(lambda x: str(x).replace('年建造', ''))
        data['property_area'] = data['property_area'].apply(lambda x: str(x).replace('㎡', ''))
        data['property_price'] = data['property_price'].apply(lambda x: str(x).replace('万', ''))
        data['property_average_price'] = data['property_average_price'].apply(lambda x: str(x).replace('元/㎡', ''))
        data['property_bedroom'] = data['property_info'].apply(lambda x: x[0])
        data['property_livingroom'] = data['property_info'].apply(lambda x: x[2])
        data['property_lavatory'] = data['property_info'].apply(lambda x: x[4])
        data['property_height'] = data['property_floor'].apply(lambda x: re.search(r'\d+', x).group())
        data['property_floor'] = data['property_floor'].apply(lambda x: x[:x.find('(')] if x.find('(') != -1 else x)
        data['property_loc'] = data['property_loc'].apply(lambda x: str(x).split('-', 1)[0])
        data['property_price'] = data['property_price'].apply(lambda x: float(x) * 10000)
        data.to_csv(f'./data_statistic/{filename}')
        sum_df = sum_df.append(data, ignore_index=True)

    title_df = sum_df['title']
    title_list = title_df.to_list()
    save_txt(title_list)
    sum_df.to_csv(f'./data_statistic/SJZ_Property.csv')


def save_txt(text_list):
    with open('./text/title.txt', 'w', encoding='utf-8') as f:
        for item in text_list:
            f.write(item + '\n')
    f.close()


def clean():
    find_csv('./', filelist=file_list)

    data_cleaning('./', filelist=file_list)

    data_statistic('./', filelist=file_list)
